#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
'''
Created on 2016年5月23日
@author: zhaoxiang
'''
import sys,os
sys.path.append('/home/hadoop/udf_data/python/user_track')
os.environ['PYTHON_EGG_CACHE'] = '/tmp/hadoop/.python-eggs/'
os.environ['PYTHON_EGG_DIR']='/tmp/hadoop/.python-eggs/'
import time,datetime,format_time
import json
import jieba 
import config
from config import  DICT_FILE
from statistic_city_country import ProductTagger
from DB import *
import gua
from util_func import dateToTimestamp
# from util_func import strdecode
# from user_util import black_hotel_users
from MongoClient import MongoClient

reload(sys)
sys.setdefaultencoding("utf8")

doraid_views={}

#==============================
#折扣
#===============================
def buy_lm(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select uid, lid from qyer_orderform.orderform where payment = 1 and datetime >= %s and datetime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result["uid"]
        lid = result["lid"]
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if uid_result : 
            dora_id = uid_result["dora_id"]
            if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[lid], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
            else : 
                doraid_views[dora_id]["buy_lm"].append(lid)
                doraid_views[dora_id]["uids"].add(uid)
    
    return doraid_views

#==============================
#酒店预订
#===============================
black_hotel_users = set([191066, 4954313, 2311611, 5397676, 1569414, 4935486])         
def book_hotel(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select id, booker_hotel_id, uid, guid from business_partner_hotel_booking_order where book_time >= %s and book_time < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        booking_hotel_id = result["booker_hotel_id"]
        qyer_db.execute("select id from business_hotel where booking_hotel_id = %s", (booking_hotel_id,))
        hotel_result = qyer_db.fetchone()
        if not hotel_result : continue
        hotel_id = hotel_result["id"]

        uid = result["uid"]
        if uid in black_hotel_users : continue
        guid = result["guid"]
        dora_id = 0
        if uid :
            uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
            if uid_result : dora_id = uid_result["dora_id"]
        if len(guid) == 36 and dora_id == 0 : 
            guid_result = mongo_db.execute("guid_doraid_1", {"_id":guid}, "selectone")
            if guid_result : dora_id = guid_result["dora_id"]

        if dora_id :
            if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
            if uid : doraid_views[dora_id]["uids"].add(uid)
            doraid_views[dora_id]["book_hotel"].append(hotel_id)
        
    qyer_db.close()
    return doraid_views

#==============================
#锦囊下载
#===============================
def down_guide(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid, jid from jn_down where downtime > %s and downtime <= %s and uid > 0", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['uid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if uid_result : 
            dora_id = uid_result["dora_id"]
            if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
            doraid_views[dora_id]["d_guide"].add(result["jid"])
            if uid : doraid_views[dora_id]["uids"].add(uid)
    return doraid_views

#==============================
#搜索
#===============================
def ask_search(starttime, endtime, doraid_views) :
    jieba.load_userdict(DICT_FILE) 
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid, search from ask_search_log where addtime >= %s and addtime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['uid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        segs = jieba.cut(result['search'])
        words = [w.encode('utf8', 'ignore') for w in segs if len(w) > 1]
        if len(words) : 
            if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
            if uid > 0 : doraid_views[dora_id]["uids"].add(uid)
            doraid_views[dora_id]["search"]["wd"] += words
    return doraid_views

#==============================
#提问
#===============================
def ask_question(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select qid, uid from qyer.ask_question where addtime >= %s and addtime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['uid']
        qid = result['qid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[qid], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else : 
            doraid_views[dora_id]["ask"].append(qid)
            if uid > 0 : doraid_views[dora_id]["uids"].add(uid)
    return doraid_views

#==============================
#问答
#===============================
def answer_question(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select qid, uid from qyer.ask_answer where addtime >= %s and addtime < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result["uid"]
        qid = result["qid"]
        qyer_db.execute("select uid from qyer.ask_question where qid = %s", (qid,))
        q_result = qyer_db.fetchone()
        if not q_result : continue
        if q_result["uid"] == uid : continue
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([qid]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else :
            doraid_views[dora_id]["answer"].add(qid)
            if uid : doraid_views[dora_id]["uids"].add(uid)
    qyer_db.close()
    return doraid_views

#==============================
#点评
#===============================
def place_comment(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid, cityid from qyer.place_comment where datetime > %s and datetime <= %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result['uid']
        cityid = result['cityid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[cityid], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else :
            doraid_views[dora_id]["p_comment"]["city"].append(cityid)
            if uid > 0 : doraid_views[dora_id]["uids"].add(uid)

    qyer_db.execute("select uid, countryid from qyer.place_comment_country where datetime > %s and datetime <= %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result['uid']
        countryid = result['countryid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[countryid], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else :
            doraid_views[dora_id]["p_comment"]["country"].append(countryid)
            if uid > 0 : doraid_views[dora_id]["uids"].add(uid)

    qyer_db.execute("select uid, poiid from qyer.poi_comment where datetime > %s and datetime <= %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    for result in result_set :
        uid = result['uid']
        poiid = result['poiid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[poiid]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else :
            doraid_views[dora_id]["p_comment"]["poi"].append(poiid)
            if uid > 0 : doraid_views[dora_id]["uids"].add(uid)

    qyer_db.close()
    return doraid_views

#==============================
#论坛
#===============================
def coplay_thread(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
    qyer_db.execute("select uid as authorid, id as tid from bbs.threads where type = 3 and created >= %s and created < %s", (starttime, endtime))
    #qyer_db.execute("select authorid, tid from newbbs_utf8.cdb_threads where typeid = 8 and dateline >= %s and dateline < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result['authorid']
        tid = result['tid']
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[tid], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else :
            doraid_views[dora_id]["coplay"].append(tid)
            if uid : doraid_views[dora_id]["uids"].add(uid)

    return doraid_views

def travel_thread(starttime, endtime, doraid_views) :
    qyer_db = DB(gua.conn_qyer["host"], gua.conn_qyer["username"], gua.conn_qyer["password"], gua.conn_qyer["db"], gua.conn_qyer["port"])
    qyer_db.execute("select uid as authorid, id as tid from bbs.threads where type in (1,2) and created >= %s and created < %s", (starttime, endtime))
    #qyer_db.execute("select authorid, tid from newbbs_utf8.cdb_threads where typeid = 5 and dateline >= %s and dateline < %s", (starttime, endtime))
    result_set = qyer_db.fetchall()
    qyer_db.close()
    for result in result_set :
        uid = result["authorid"]
        tid = result["tid"]
        uid_result = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
        if not uid_result : continue
        dora_id = uid_result["dora_id"]
        if dora_id not in doraid_views : doraid_views[dora_id] = {"uids":set([uid]), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[tid], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
        else :
            doraid_views[dora_id]["p_travel"].append(tid)
            if uid > 0 : doraid_views[dora_id]["uids"].add(uid)
    return doraid_views

def tag_doraid_action(timeStamp, doraid_views) :
    pt = ProductTagger()
    ra_db = DB(gua.conn_ra["host"], gua.conn_ra["username"], gua.conn_ra["password"], gua.conn_ra["db"], gua.conn_ra["port"])
    print len(doraid_views)
    timeArray = time.localtime(timeStamp)
    strDate = time.strftime("%Y%m%d", timeArray)
  
    with open(os.path.join(config.product_action_outputDir,"out.%s" % strDate),"w+") as f:
        for dora_id, views in doraid_views.iteritems() :
            uids = list(views["uids"])
            del views["uids"]
    #         dora_result = mongo_db.execute("dora_family_1", {"_id":dora_id}, "selectone")
    #         if dora_result :
    #             if len(dora_result["guids"]) : guid = dora_result["guids"][-1]
    #             else : guid = ""
    #             if len(dora_result["device_ids"]) : deviceid = dora_result["device_ids"][-1]
    #             else : deviceid = ""
    #             if len(uids) == 0 : uids = dora_result["uids"]
    #         else : continue
    #         action_tags = {}
    #         action_content = {}
            for parsed_type, parsed_content in views.iteritems() :
                if parsed_type == 'search':
                    continue 
                if len(parsed_content) == 0 :
                    continue 
                parsed_tags = pt.product_tag(parsed_type, parsed_content)
                if len(parsed_tags.get("country", {})) == 0 and len(parsed_tags.get("city", {})) == 0 : 
                    continue
                if(type(parsed_content) == set):
                    parsed_content=list(parsed_content)
                f.write("%s\t%s\t%s\t%s\t%s\n" % (dora_id,json.dumps(uids),parsed_type,json.dumps(parsed_content),json.dumps(parsed_tags)))
    ra_db.close()

def dump_user_action(theday) :
    doraid_views = {}
    starttime = dateToTimestamp(theday)
#     先删除旧数据
    delete_data(starttime)
    endtime = starttime + 60*60*24
    print format_time.get_now(), "======begin deal with %s=====" % theday
    print format_time.get_now(), "buy lm"
    doraid_views = buy_lm(starttime, endtime, doraid_views)
    print format_time.get_now(), "book hotel"
    doraid_views = book_hotel(starttime, endtime, doraid_views)
    print format_time.get_now(), "download guide"
    doraid_views = down_guide(starttime, endtime, doraid_views)
    print format_time.get_now(), "ask search"
    doraid_views = ask_search(starttime, endtime, doraid_views)
    print format_time.get_now(), "ask question"
    doraid_views = ask_question(starttime, endtime, doraid_views)
    print format_time.get_now(), "answer question"
    doraid_views = answer_question(starttime, endtime, doraid_views)
    print format_time.get_now(), "place comment"
    doraid_views = place_comment(starttime, endtime, doraid_views)
    print format_time.get_now(), "coplay thread"
    doraid_views = coplay_thread(starttime, endtime, doraid_views)
    print format_time.get_now(), "post travel thread"
    doraid_views = travel_thread(starttime, endtime, doraid_views)
    print format_time.get_now(), "tag doraid action"
    tag_doraid_action(starttime, doraid_views)
    print format_time.get_now(), "======end deal with %s=====" % theday
    
def delete_data(theday) :
    ra_db = DB(gua.conn_ra["host"], gua.conn_ra["username"], gua.conn_ra["password"], gua.conn_ra["db"], gua.conn_ra["port"])
#     ra_db.execute("delete from track.placetag_log1 where timestamp = %s", (theday,))
    ra_db.close() 
        

def main(*argv) :
    if len(argv) == 1 :
        startDay = datetime.datetime.now()
        startDay = datetime.datetime(startDay.year, startDay.month, startDay.day)
        dump_user_action(startDay)
    elif len(argv) == 2 :
        startDay = datetime.datetime.strptime(argv[1], "%Y%m%d")
        dump_user_action(startDay)
    elif len(argv) == 3 :
        startDay = datetime.datetime.strptime(argv[1], "%Y%m%d")
        endDay = datetime.datetime.strptime(argv[2], "%Y%m%d")
        while startDay < endDay :
            dump_user_action(startDay)
            startDay += datetime.timedelta(days = 1)

mongo_db = MongoClient(gua.conn_mongo["host"], gua.conn_mongo["db"])

if __name__ == '__main__' :
    main(*sys.argv)
    mongo_db.close()